package main

import (
	"context"
	"crawler/conf"
	duplicateRpc "crawler/grpc/duplicate-rpc"
	"google.golang.org/grpc"
	"io/ioutil"
	"log"
	"net/http"
	"regexp"
	"time"
)

var arrs = []string{
	"https://www.csdn.net/",
	"https://www.csdn.net/nav/career",
	//"https://www.csdn.net/nav/python",
	//"https://www.csdn.net/nav/java",
	//"https://www.csdn.net/nav/web",
	//"https://www.csdn.net/nav/arch",
	//"https://www.csdn.net/nav/db",
	//"https://www.csdn.net/nav/game",
	//"https://www.csdn.net/nav/mobile",
	//"https://www.csdn.net/nav/ops",
	//"https://www.csdn.net/nav/fund",
	//"https://www.csdn.net/nav/avi",
	//"https://www.csdn.net/nav/other",
}

var userRe = regexp.MustCompile(conf.OTHER_COMPILE)

func fetchAndParse(url string) []string {
	res := []string{}
	resp, err := http.Get(url)
	if err != nil {
		log.Fatal("get-userId.fetch error=", err)
		return res
	}
	defer resp.Body.Close()

	if resp.StatusCode == http.StatusOK {
		content, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			log.Fatal("get-userId.readAll error=", err)
			return res
		}

		submatch := userRe.FindAllSubmatch(content, -1)
		for _, v := range submatch {
			res = append(res, string(v[1]))
		}
	}
	return res
}

func sendToRds(users []string) {
	// 连接rpc通道
	conn, err := grpc.Dial("172.20.159.204:28080", grpc.WithInsecure())
	if err != nil {
		log.Fatalln(err)
	}
	defer conn.Close()

	// 初始化客户端
	cli := duplicateRpc.NewDuplicateClient(conn)

	// 调用方法
	reqBody := &duplicateRpc.SendUserRequest{Users: users}
	res := &duplicateRpc.SendUserReply{Msg: ""}
	res, _ = cli.SendUsers(context.Background(), reqBody)
	log.Println(res.Msg)
}

func main() {
	for _, arr := range arrs {
		resp := fetchAndParse(arr)

		// 非空才发送给redis去重模块
		if len(resp) > 0 {
			// 发送给redis去重模块
			go sendToRds(resp)
		}
		// 每个网址之间间隔5秒
		time.Sleep(5 * time.Second)
	}
}
